{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_on4sre8",
    "id": "7572198A886042739E2AD4ABD4B8B745",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "# 梯度下降 \n",
    "（[Boyd & Vandenberghe, 2004](https://d2l.ai/chapter_references/zreferences.html#boyd-vandenberghe-2004)）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "graffitiCellId": "id_nl4rnln",
    "id": "BF05AB89B8E640C281B914AAFA206BB0",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import torch\n",
    "import time\n",
    "from torch import nn, optim\n",
    "import math\n",
    "import sys\n",
    "sys.path.append('/home/kesci/input')\n",
    "import d2lzh1981 as d2l"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_t1q0ber",
    "id": "BCD65556302D4CCF8FF96FE8ED6623D2",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "## 一维梯度下降"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_pppxq5z",
    "id": "94EC994AB87740408746EC4E07170286",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "**证明：沿梯度反方向移动自变量可以减小函数值**\n",
    "\n",
    "泰勒展开：\n",
    "\n",
    "$$\n",
    "f(x+\\epsilon)=f(x)+\\epsilon f^{\\prime}(x)+\\mathcal{O}\\left(\\epsilon^{2}\\right)\n",
    "$$\n",
    "\n",
    "代入沿梯度方向的移动量 $\\eta f^{\\prime}(x)$：\n",
    "\n",
    "$$\n",
    "f\\left(x-\\eta f^{\\prime}(x)\\right)=f(x)-\\eta f^{\\prime 2}(x)+\\mathcal{O}\\left(\\eta^{2} f^{\\prime 2}(x)\\right)\n",
    "$$\n",
    "\n",
    "$$\n",
    "f\\left(x-\\eta f^{\\prime}(x)\\right) \\lesssim f(x)\n",
    "$$\n",
    "\n",
    "\n",
    "$$\n",
    "x \\leftarrow x-\\eta f^{\\prime}(x)\n",
    "$$\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_mhywc3s",
    "id": "E74B71FF3F134AFE89F6C3DD34026F32",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "e.g.\n",
    "\n",
    "$$\n",
    "f(x) = x^2\n",
    "$$\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "graffitiCellId": "id_6xl9gas",
    "id": "842E5A0DE2274C24A6AF587E490517FD",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 10, x: 0.06046617599999997\n"
     ]
    }
   ],
   "source": [
    "def f(x):\n",
    "    return x**2  # Objective function\n",
    "\n",
    "def gradf(x):\n",
    "    return 2 * x  # Its derivative\n",
    "\n",
    "def gd(eta):\n",
    "    x = 10\n",
    "    results = [x]\n",
    "    for i in range(10):\n",
    "        x -= eta * gradf(x)\n",
    "        results.append(x)\n",
    "    print('epoch 10, x:', x)\n",
    "    return results\n",
    "\n",
    "res = gd(0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "graffitiCellId": "id_bpk707t",
    "id": "474742EF0BCD428780FEBB6BA93E1785",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<img src=\"https://cdn.kesci.com/rt_upload/474742EF0BCD428780FEBB6BA93E1785/q5oim9v02y.svg\">"
      ],
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def show_trace(res):\n",
    "    n = max(abs(min(res)), abs(max(res)))\n",
    "    f_line = np.arange(-n, n, 0.01)\n",
    "    d2l.set_figsize((3.5, 2.5))\n",
    "    d2l.plt.plot(f_line, [f(x) for x in f_line],'-')\n",
    "    d2l.plt.plot(res, [f(x) for x in res],'-o')\n",
    "    d2l.plt.xlabel('x')\n",
    "    d2l.plt.ylabel('f(x)')\n",
    "    \n",
    "\n",
    "show_trace(res)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_mhgaljc",
    "id": "3A083FD82CE549C38C1B76F74672A80C",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "### 学习率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "graffitiCellId": "id_8nd5ipw",
    "id": "63F2E327184549D38E1C1323766EBA35",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 10, x: 3.4867844009999995\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://cdn.kesci.com/rt_upload/63F2E327184549D38E1C1323766EBA35/q5oim918r4.svg\">"
      ],
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_trace(gd(0.05))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "graffitiCellId": "id_lqgn6wo",
    "id": "540302EC5E434318809B895B453AABE4",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 10, x: 61.917364224000096\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://cdn.kesci.com/rt_upload/540302EC5E434318809B895B453AABE4/q5oim98mjm.svg\">"
      ],
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_trace(gd(1.1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_8hf2rdv",
    "id": "2EC285A41CCE4994BB9AFDEAFF9A7283",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "### 局部极小值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_32accll",
    "id": "1D6CF0E790DF412B8BEF0C105B28E908",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "e.g.\n",
    "\n",
    "$$\n",
    "f(x) = x\\cos cx\n",
    "$$\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "graffitiCellId": "id_rq7w3yx",
    "id": "8372CFD30F534D63B847DED05E6BC1E7",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 10, x: -1.528165927635083\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://cdn.kesci.com/rt_upload/8372CFD30F534D63B847DED05E6BC1E7/q5oim9yozt.svg\">"
      ],
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "c = 0.15 * np.pi\n",
    "\n",
    "def f(x):\n",
    "    return x * np.cos(c * x)\n",
    "\n",
    "def gradf(x):\n",
    "    return np.cos(c * x) - c * x * np.sin(c * x)\n",
    "\n",
    "show_trace(gd(2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_nf23exa",
    "id": "534F5C33B2D14E7083455E6F9189AED1",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "## 多维梯度下降"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_e2tln4o",
    "id": "99DE5DDC9BFB4A638E07224320A1F21F",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "\n",
    "$$\n",
    "\\nabla f(\\mathbf{x})=\\left[\\frac{\\partial f(\\mathbf{x})}{\\partial x_{1}}, \\frac{\\partial f(\\mathbf{x})}{\\partial x_{2}}, \\dots, \\frac{\\partial f(\\mathbf{x})}{\\partial x_{d}}\\right]^{\\top}\n",
    "$$\n",
    "\n",
    "$$\n",
    "f(\\mathbf{x}+\\epsilon)=f(\\mathbf{x})+\\epsilon^{\\top} \\nabla f(\\mathbf{x})+\\mathcal{O}\\left(\\|\\epsilon\\|^{2}\\right)\n",
    "$$\n",
    "\n",
    "$$\n",
    "\\mathbf{x} \\leftarrow \\mathbf{x}-\\eta \\nabla f(\\mathbf{x})\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "graffitiCellId": "id_kxz1kl7",
    "id": "D2A69BABDFD04AF48847E925A8DB8351",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def train_2d(trainer, steps=20):\n",
    "    x1, x2 = -5, -2\n",
    "    results = [(x1, x2)]\n",
    "    for i in range(steps):\n",
    "        x1, x2 = trainer(x1, x2)\n",
    "        results.append((x1, x2))\n",
    "    print('epoch %d, x1 %f, x2 %f' % (i + 1, x1, x2))\n",
    "    return results\n",
    "\n",
    "def show_trace_2d(f, results): \n",
    "    d2l.plt.plot(*zip(*results), '-o', color='#ff7f0e')\n",
    "    x1, x2 = np.meshgrid(np.arange(-5.5, 1.0, 0.1), np.arange(-3.0, 1.0, 0.1))\n",
    "    d2l.plt.contour(x1, x2, f(x1, x2), colors='#1f77b4')\n",
    "    d2l.plt.xlabel('x1')\n",
    "    d2l.plt.ylabel('x2')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_843firi",
    "id": "1319758B74AC452588639895F40862F3",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "\n",
    "$$\n",
    "f(x) = x_1^2 + 2x_2^2\n",
    "$$\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "graffitiCellId": "id_rbr2v5f",
    "id": "7C4F69BAE0E2424D80BC99B0867DB110",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 20, x1 -0.057646, x2 -0.000073\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://cdn.kesci.com/rt_upload/7C4F69BAE0E2424D80BC99B0867DB110/q5oim9cca5.svg\">"
      ],
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "eta = 0.1\n",
    "\n",
    "def f_2d(x1, x2):  # 目标函数\n",
    "    return x1 ** 2 + 2 * x2 ** 2\n",
    "\n",
    "def gd_2d(x1, x2):\n",
    "    return (x1 - eta * 2 * x1, x2 - eta * 4 * x2)\n",
    "\n",
    "show_trace_2d(f_2d, train_2d(gd_2d))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_ojs9x5n",
    "id": "F1AFA19516FE42F3827F10E5F5B4BDD0",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "## 自适应方法"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_ybiw4cv",
    "id": "98F16E9A992D4D189DE9B5E21F70B750",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "### 牛顿法"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_s76eff5",
    "id": "62C9EFD5B9D94D058B395F93EF9162A7",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "在 $x + \\epsilon$ 处泰勒展开：\n",
    "\n",
    "$$\n",
    "f(\\mathbf{x}+\\epsilon)=f(\\mathbf{x})+\\epsilon^{\\top} \\nabla f(\\mathbf{x})+\\frac{1}{2} \\epsilon^{\\top} \\nabla \\nabla^{\\top} f(\\mathbf{x}) \\epsilon+\\mathcal{O}\\left(\\|\\epsilon\\|^{3}\\right)\n",
    "$$\n",
    "\n",
    "最小值点处满足: $\\nabla f(\\mathbf{x})=0$, 即我们希望 $\\nabla f(\\mathbf{x} + \\epsilon)=0$, 对上式关于 $\\epsilon$ 求导，忽略高阶无穷小，有：\n",
    "\n",
    "$$\n",
    "\\nabla f(\\mathbf{x})+\\boldsymbol{H}_{f} \\boldsymbol{\\epsilon}=0 \\text { and hence } \\epsilon=-\\boldsymbol{H}_{f}^{-1} \\nabla f(\\mathbf{x})\n",
    "$$\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "graffitiCellId": "id_ne0bb4i",
    "id": "93AB8E85F6264E129E9F622747D059FD",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 10, x: 0.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://cdn.kesci.com/rt_upload/93AB8E85F6264E129E9F622747D059FD/q5oima7f51.svg\">"
      ],
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "c = 0.5\n",
    "\n",
    "def f(x):\n",
    "    return np.cosh(c * x)  # Objective\n",
    "\n",
    "def gradf(x):\n",
    "    return c * np.sinh(c * x)  # Derivative\n",
    "\n",
    "def hessf(x):\n",
    "    return c**2 * np.cosh(c * x)  # Hessian\n",
    "\n",
    "# Hide learning rate for now\n",
    "def newton(eta=1):\n",
    "    x = 10\n",
    "    results = [x]\n",
    "    for i in range(10):\n",
    "        x -= eta * gradf(x) / hessf(x)\n",
    "        results.append(x)\n",
    "    print('epoch 10, x:', x)\n",
    "    return results\n",
    "\n",
    "show_trace(newton())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "graffitiCellId": "id_0zttur6",
    "id": "6F40E126C3FD4E059463B6B5E1B809E5",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 10, x: 26.83413291324767\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://cdn.kesci.com/rt_upload/6F40E126C3FD4E059463B6B5E1B809E5/q5oimahwv7.svg\">"
      ],
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "c = 0.15 * np.pi\n",
    "\n",
    "def f(x):\n",
    "    return x * np.cos(c * x)\n",
    "\n",
    "def gradf(x):\n",
    "    return np.cos(c * x) - c * x * np.sin(c * x)\n",
    "\n",
    "def hessf(x):\n",
    "    return - 2 * c * np.sin(c * x) - x * c**2 * np.cos(c * x)\n",
    "\n",
    "show_trace(newton())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "graffitiCellId": "id_x3tkks0",
    "id": "5FE9DDAA901B422181ECFBFB171DDDF0",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 10, x: 7.269860168684531\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://cdn.kesci.com/rt_upload/5FE9DDAA901B422181ECFBFB171DDDF0/q5oimau6rq.svg\">"
      ],
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_trace(newton(0.5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_abhcxnu",
    "id": "585574DCA7F24BF682E485EDB1EE6B3E",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "### 收敛性分析"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_p7m23hf",
    "id": "BF48B399382743CABB1D0ABCD94B3144",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "只考虑在函数为凸函数, 且最小值点上 $f''(x^*) > 0$ 时的收敛速度：\n",
    "\n",
    "令 $x_k$ 为第 $k$ 次迭代后 $x$ 的值， $e_{k}:=x_{k}-x^{*}$ 表示 $x_k$ 到最小值点 $x^{*}$ 的距离，由 $f'(x^{*}) = 0$:\n",
    "\n",
    "$$\n",
    "0=f^{\\prime}\\left(x_{k}-e_{k}\\right)=f^{\\prime}\\left(x_{k}\\right)-e_{k} f^{\\prime \\prime}\\left(x_{k}\\right)+\\frac{1}{2} e_{k}^{2} f^{\\prime \\prime \\prime}\\left(\\xi_{k}\\right) \\text{for some } \\xi_{k} \\in\\left[x_{k}-e_{k}, x_{k}\\right]\n",
    "$$\n",
    "\n",
    "两边除以 $f''(x_k)$, 有：\n",
    "\n",
    "$$\n",
    "e_{k}-f^{\\prime}\\left(x_{k}\\right) / f^{\\prime \\prime}\\left(x_{k}\\right)=\\frac{1}{2} e_{k}^{2} f^{\\prime \\prime \\prime}\\left(\\xi_{k}\\right) / f^{\\prime \\prime}\\left(x_{k}\\right)\n",
    "$$\n",
    "\n",
    "代入更新方程 $x_{k+1} = x_{k} - f^{\\prime}\\left(x_{k}\\right) / f^{\\prime \\prime}\\left(x_{k}\\right)$, 得到：\n",
    "\n",
    "$$\n",
    "x_k - x^{*} - f^{\\prime}\\left(x_{k}\\right) / f^{\\prime \\prime}\\left(x_{k}\\right) =\\frac{1}{2} e_{k}^{2} f^{\\prime \\prime \\prime}\\left(\\xi_{k}\\right) / f^{\\prime \\prime}\\left(x_{k}\\right)\n",
    "$$\n",
    "\n",
    "\n",
    "$$\n",
    "x_{k+1} - x^{*} = e_{k+1} = \\frac{1}{2} e_{k}^{2} f^{\\prime \\prime \\prime}\\left(\\xi_{k}\\right) / f^{\\prime \\prime}\\left(x_{k}\\right)\n",
    "$$\n",
    "\n",
    "当 $\\frac{1}{2} f^{\\prime \\prime \\prime}\\left(\\xi_{k}\\right) / f^{\\prime \\prime}\\left(x_{k}\\right) \\leq c$ 时，有:\n",
    "\n",
    "$$\n",
    "e_{k+1} \\leq c e_{k}^{2}\n",
    "$$\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_3uu8q0a",
    "id": "650317A0356D4B23A2A9D3B22464D7CC",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "### 预处理 （Heissan阵辅助梯度下降）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_vqv7thj",
    "id": "12635895E5334461A0ED724BFD885B25",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "\n",
    "$$\n",
    "\\mathbf{x} \\leftarrow \\mathbf{x}-\\eta \\operatorname{diag}\\left(H_{f}\\right)^{-1} \\nabla \\mathbf{x}\n",
    "$$\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_8khl2xc",
    "id": "34301E3820F5435795E8DC14A663FD87",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "### 梯度下降与线性搜索（共轭梯度法）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_cvo62el",
    "id": "882E8525B5E64B159A284225BB043DAF",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "# 随机梯度下降"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_5myc4ii",
    "id": "BE0C412BB3A9439BA8C064ECA4239857",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "## 随机梯度下降参数更新"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_vccc2dg",
    "id": "CEDA7E740EC34520859761F73D38F3C8",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "对于有 $n$ 个样本对训练数据集，设 $f_i(x)$ 是第 $i$ 个样本的损失函数, 则目标函数为:\n",
    "\n",
    "$$\n",
    "f(\\mathbf{x})=\\frac{1}{n} \\sum_{i=1}^{n} f_{i}(\\mathbf{x})\n",
    "$$\n",
    "\n",
    "其梯度为:\n",
    "\n",
    "$$\n",
    "\\nabla f(\\mathbf{x})=\\frac{1}{n} \\sum_{i=1}^{n} \\nabla f_{i}(\\mathbf{x})\n",
    "$$\n",
    "\n",
    "使用该梯度的一次更新的时间复杂度为 $\\mathcal{O}(n)$\n",
    "\n",
    "随机梯度下降更新公式 $\\mathcal{O}(1)$:\n",
    "\n",
    "$$\n",
    "\\mathbf{x} \\leftarrow \\mathbf{x}-\\eta \\nabla f_{i}(\\mathbf{x})\n",
    "$$\n",
    "\n",
    "且有：\n",
    "\n",
    "$$\n",
    "\\mathbb{E}_{i} \\nabla f_{i}(\\mathbf{x})=\\frac{1}{n} \\sum_{i=1}^{n} \\nabla f_{i}(\\mathbf{x})=\\nabla f(\\mathbf{x})\n",
    "$$\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_xa873bz",
    "id": "9F69CFFA97964916B90CD2BB23B01ABE",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "e.g. \n",
    "\n",
    "$$\n",
    "f(x_1, x_2) = x_1^2 + 2 x_2^2\n",
    "$$\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "graffitiCellId": "id_dccr2u1",
    "id": "4CB5541B156E4F0AB7E145B54C8E6EA0",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 50, x1 -0.027566, x2 0.137605\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://cdn.kesci.com/rt_upload/4CB5541B156E4F0AB7E145B54C8E6EA0/q5oimaluhh.svg\">"
      ],
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def f(x1, x2):\n",
    "    return x1 ** 2 + 2 * x2 ** 2  # Objective\n",
    "\n",
    "def gradf(x1, x2):\n",
    "    return (2 * x1, 4 * x2)  # Gradient\n",
    "\n",
    "def sgd(x1, x2):  # Simulate noisy gradient\n",
    "    global lr  # Learning rate scheduler\n",
    "    (g1, g2) = gradf(x1, x2)  # Compute gradient\n",
    "    (g1, g2) = (g1 + np.random.normal(0.1), g2 + np.random.normal(0.1))\n",
    "    eta_t = eta * lr()  # Learning rate at time t\n",
    "    return (x1 - eta_t * g1, x2 - eta_t * g2)  # Update variables\n",
    "\n",
    "eta = 0.1\n",
    "lr = (lambda: 1)  # Constant learning rate\n",
    "show_trace_2d(f, train_2d(sgd, steps=50))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_u5xwp88",
    "id": "B969ED17218C4BA887090B4B1C41D0B6",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "## 动态学习率"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_dh8duv0",
    "id": "68DC0E0518E64F20915A7FAE1489A72B",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "\n",
    "$$\n",
    "\\begin{array}{ll}{\\eta(t)=\\eta_{i} \\text { if } t_{i} \\leq t \\leq t_{i+1}} & {\\text { piecewise constant }} \\\\ {\\eta(t)=\\eta_{0} \\cdot e^{-\\lambda t}} & {\\text { exponential }} \\\\ {\\eta(t)=\\eta_{0} \\cdot(\\beta t+1)^{-\\alpha}} & {\\text { polynomial }}\\end{array}\n",
    "$$\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "graffitiCellId": "id_d78lmwl",
    "id": "7E2E2CF3EB3A4A2D90F41A070E1D008F",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1000, x1 -0.677947, x2 -0.089379\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://cdn.kesci.com/rt_upload/7E2E2CF3EB3A4A2D90F41A070E1D008F/q5oimas9qm.svg\">"
      ],
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def exponential():\n",
    "    global ctr\n",
    "    ctr += 1\n",
    "    return math.exp(-0.1 * ctr)\n",
    "\n",
    "ctr = 1\n",
    "lr = exponential  # Set up learning rate\n",
    "show_trace_2d(f, train_2d(sgd, steps=1000))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "graffitiCellId": "id_4sql34e",
    "id": "B3DF53CD6E1A458B836D2308F8263B58",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 50, x1 -0.095244, x2 -0.041674\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://cdn.kesci.com/rt_upload/B3DF53CD6E1A458B836D2308F8263B58/q5oimapo7u.svg\">"
      ],
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def polynomial():\n",
    "    global ctr\n",
    "    ctr += 1\n",
    "    return (1 + 0.1 * ctr)**(-0.5)\n",
    "\n",
    "ctr = 1\n",
    "lr = polynomial  # Set up learning rate\n",
    "show_trace_2d(f, train_2d(sgd, steps=50))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_zwblf6k",
    "id": "45094D70C75448CE8483D4ECC50DCECF",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "# 小批量随机梯度下降"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_jero8pd",
    "id": "CCED44DA669F40EB83770D4FCE98C82A",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "## 读取数据\n",
    "[读取数据](https://archive.ics.uci.edu/ml/datasets/Airfoil+Self-Noise)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "graffitiCellId": "id_f3mc1re",
    "id": "0C848C4C23DC489FACA6EC84A8EDE071",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1500, 5])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_data_ch7():  # 本函数已保存在d2lzh_pytorch包中方便以后使用\n",
    "    data = np.genfromtxt('/home/kesci/input/airfoil4755/airfoil_self_noise.dat', delimiter='\\t')\n",
    "    data = (data - data.mean(axis=0)) / data.std(axis=0) # 标准化\n",
    "    return torch.tensor(data[:1500, :-1], dtype=torch.float32), \\\n",
    "           torch.tensor(data[:1500, -1], dtype=torch.float32) # 前1500个样本(每个样本5个特征)\n",
    "\n",
    "features, labels = get_data_ch7()\n",
    "features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "graffitiCellId": "id_t2q026w",
    "id": "E7B13A69CE92494088087B62DB85D091",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>800</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3048</td>\n",
       "      <td>71.3</td>\n",
       "      <td>0.002663</td>\n",
       "      <td>126.201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3048</td>\n",
       "      <td>71.3</td>\n",
       "      <td>0.002663</td>\n",
       "      <td>125.201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1250</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3048</td>\n",
       "      <td>71.3</td>\n",
       "      <td>0.002663</td>\n",
       "      <td>125.951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1600</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3048</td>\n",
       "      <td>71.3</td>\n",
       "      <td>0.002663</td>\n",
       "      <td>127.591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3048</td>\n",
       "      <td>71.3</td>\n",
       "      <td>0.002663</td>\n",
       "      <td>127.461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>2500</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3048</td>\n",
       "      <td>71.3</td>\n",
       "      <td>0.002663</td>\n",
       "      <td>125.571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>3150</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3048</td>\n",
       "      <td>71.3</td>\n",
       "      <td>0.002663</td>\n",
       "      <td>125.201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>4000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3048</td>\n",
       "      <td>71.3</td>\n",
       "      <td>0.002663</td>\n",
       "      <td>123.061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>5000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3048</td>\n",
       "      <td>71.3</td>\n",
       "      <td>0.002663</td>\n",
       "      <td>121.301</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>6300</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3048</td>\n",
       "      <td>71.3</td>\n",
       "      <td>0.002663</td>\n",
       "      <td>119.541</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      0    1       2     3         4        5\n",
       "0   800  0.0  0.3048  71.3  0.002663  126.201\n",
       "1  1000  0.0  0.3048  71.3  0.002663  125.201\n",
       "2  1250  0.0  0.3048  71.3  0.002663  125.951\n",
       "3  1600  0.0  0.3048  71.3  0.002663  127.591\n",
       "4  2000  0.0  0.3048  71.3  0.002663  127.461\n",
       "5  2500  0.0  0.3048  71.3  0.002663  125.571\n",
       "6  3150  0.0  0.3048  71.3  0.002663  125.201\n",
       "7  4000  0.0  0.3048  71.3  0.002663  123.061\n",
       "8  5000  0.0  0.3048  71.3  0.002663  121.301\n",
       "9  6300  0.0  0.3048  71.3  0.002663  119.541"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv('/home/kesci/input/airfoil4755/airfoil_self_noise.dat', delimiter='\\t', header=None)\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_ryefk8w",
    "id": "521D2D54C45848008F28AC827A14071C",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "## 从零开始实现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "graffitiCellId": "id_wpr9jbt",
    "id": "5D8CD5C713ED4A0C828092D8E6CEE3D2",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def sgd(params, states, hyperparams):\n",
    "    for p in params:\n",
    "        p.data -= hyperparams['lr'] * p.grad.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "graffitiCellId": "id_iqqi7qi",
    "id": "3A8DB851F2CC4CA98C398C64DA8C0084",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 本函数已保存在d2lzh_pytorch包中方便以后使用\n",
    "def train_ch7(optimizer_fn, states, hyperparams, features, labels,\n",
    "              batch_size=10, num_epochs=2):\n",
    "    # 初始化模型\n",
    "    net, loss = d2l.linreg, d2l.squared_loss\n",
    "    \n",
    "    w = torch.nn.Parameter(torch.tensor(np.random.normal(0, 0.01, size=(features.shape[1], 1)), dtype=torch.float32),\n",
    "                           requires_grad=True)\n",
    "    b = torch.nn.Parameter(torch.zeros(1, dtype=torch.float32), requires_grad=True)\n",
    "\n",
    "    def eval_loss():\n",
    "        return loss(net(features, w, b), labels).mean().item()\n",
    "\n",
    "    ls = [eval_loss()]\n",
    "    data_iter = torch.utils.data.DataLoader(\n",
    "        torch.utils.data.TensorDataset(features, labels), batch_size, shuffle=True)\n",
    "    \n",
    "    for _ in range(num_epochs):\n",
    "        start = time.time()\n",
    "        for batch_i, (X, y) in enumerate(data_iter):\n",
    "            l = loss(net(X, w, b), y).mean()  # 使用平均损失\n",
    "            \n",
    "            # 梯度清零\n",
    "            if w.grad is not None:\n",
    "                w.grad.data.zero_()\n",
    "                b.grad.data.zero_()\n",
    "                \n",
    "            l.backward()\n",
    "            optimizer_fn([w, b], states, hyperparams)  # 迭代模型参数\n",
    "            if (batch_i + 1) * batch_size % 100 == 0:\n",
    "                ls.append(eval_loss())  # 每100个样本记录下当前训练误差\n",
    "    # 打印结果和作图\n",
    "    print('loss: %f, %f sec per epoch' % (ls[-1], time.time() - start))\n",
    "    d2l.set_figsize()\n",
    "    d2l.plt.plot(np.linspace(0, num_epochs, len(ls)), ls)\n",
    "    d2l.plt.xlabel('epoch')\n",
    "    d2l.plt.ylabel('loss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "graffitiCellId": "id_de54suf",
    "id": "092890CC6C4941B38908FAA19887696A",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def train_sgd(lr, batch_size, num_epochs=2):\n",
    "    train_ch7(sgd, None, {'lr': lr}, features, labels, batch_size, num_epochs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_0juox2v",
    "id": "0F05E97E82964FF08131F509074DE433",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "对比"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "graffitiCellId": "id_de54suf",
    "id": "855ACC5ACA5A472F9BE384E668B84EB2",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.244373, 0.009881 sec per epoch\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://cdn.kesci.com/rt_upload/855ACC5ACA5A472F9BE384E668B84EB2/q5oj6bxc03.svg\">"
      ],
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_sgd(1, 1500, 6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "graffitiCellId": "id_d7d5ym4",
    "id": "223ADA73CFAB42EE87A9BFC9E6B709C2",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.245968, 0.463836 sec per epoch\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://cdn.kesci.com/rt_upload/223ADA73CFAB42EE87A9BFC9E6B709C2/q5oj6ckl5j.svg\">"
      ],
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_sgd(0.005, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "graffitiCellId": "id_g8kfyuu",
    "id": "BBA66FD2F352410F88CD4DB1FA37FCD7",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.243900, 0.065017 sec per epoch\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://cdn.kesci.com/rt_upload/BBA66FD2F352410F88CD4DB1FA37FCD7/q5oj6cw1e3.svg\">"
      ],
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_sgd(0.05, 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "graffitiCellId": "id_majp1kb",
    "id": "AB1E52EA6CEB4124800BBD99B4F21553",
    "jupyter": {},
    "mdEditEnable": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "source": [
    "## 简洁实现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "graffitiCellId": "id_adqg864",
    "id": "60C9FC5C629D49C9857BC163E384843A",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 本函数与原书不同的是这里第一个参数优化器函数而不是优化器的名字\n",
    "# 例如: optimizer_fn=torch.optim.SGD, optimizer_hyperparams={\"lr\": 0.05}\n",
    "def train_pytorch_ch7(optimizer_fn, optimizer_hyperparams, features, labels,\n",
    "                    batch_size=10, num_epochs=2):\n",
    "    # 初始化模型\n",
    "    net = nn.Sequential(\n",
    "        nn.Linear(features.shape[-1], 1)\n",
    "    )\n",
    "    loss = nn.MSELoss()\n",
    "    optimizer = optimizer_fn(net.parameters(), **optimizer_hyperparams)\n",
    "\n",
    "    def eval_loss():\n",
    "        return loss(net(features).view(-1), labels).item() / 2\n",
    "\n",
    "    ls = [eval_loss()]\n",
    "    data_iter = torch.utils.data.DataLoader(\n",
    "        torch.utils.data.TensorDataset(features, labels), batch_size, shuffle=True)\n",
    "\n",
    "    for _ in range(num_epochs):\n",
    "        start = time.time()\n",
    "        for batch_i, (X, y) in enumerate(data_iter):\n",
    "            # 除以2是为了和train_ch7保持一致, 因为squared_loss中除了2\n",
    "            l = loss(net(X).view(-1), y) / 2 \n",
    "            \n",
    "            optimizer.zero_grad()\n",
    "            l.backward()\n",
    "            optimizer.step()\n",
    "            if (batch_i + 1) * batch_size % 100 == 0:\n",
    "                ls.append(eval_loss())\n",
    "    # 打印结果和作图\n",
    "    print('loss: %f, %f sec per epoch' % (ls[-1], time.time() - start))\n",
    "    d2l.set_figsize()\n",
    "    d2l.plt.plot(np.linspace(0, num_epochs, len(ls)), ls)\n",
    "    d2l.plt.xlabel('epoch')\n",
    "    d2l.plt.ylabel('loss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "graffitiCellId": "id_c6ndach",
    "id": "0CD81A787CAA435C9EF0F8ED1208FFC4",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.243770, 0.047664 sec per epoch\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://cdn.kesci.com/rt_upload/0CD81A787CAA435C9EF0F8ED1208FFC4/q5oj6c1ds2.svg\">"
      ],
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_pytorch_ch7(optim.SGD, {\"lr\": 0.05}, features, labels, 10)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
